'''
Descripttion: 
Author: YeJianFa
Date: 2021-03-11 17:21:13
LastEditors: Please set LastEditors
LastEditTime: 2021-07-01 11:48:22
'''
import scrapy
import time,datetime
from ..items import ZdmhotbtwdateItem

# 爬取某活动热度
class ActHotSpider(scrapy.Spider):
    name = 'actHot'
    allowed_domains = ['smzdm.com']
    # start_urls = ['https://www.smzdm.com/tag/tlpl4ln/post/']
    custom_settings = {
        'ITEM_PIPELINES': {'zdmHotBtwDate.pipelines.ZdmhotbtwdatePipeline': 300}
    }

    #start_date = '2020-06-01 00:00:00'
    #end_date = '2021-06-30 23:59:59' # 2021-03-11
    data = []

    def parse(self, response):

        pipe_item = ZdmhotbtwdateItem()
        list_box = response.xpath("//div[@class='list list_exp']")
        for item in list_box:
            date_str = item.xpath("./div[@class='listRight']/div[@class='lrTop']/span[@class='lrTime']/text()").extract_first()
            if len(date_str) > 5 :
                date_str = '2021-' + date_str+':00'
            else :
                date_str = time.strftime('%Y-%m-%d')+' '+ date_str+':00'
            
            #start_date = self.start_date
            #end_date = self.end_date

            time_str = time.mktime(time.strptime(date_str,'%Y-%m-%d %H:%M:%S')) #根据时间字符获取时间戳，秒数
            #start_time = time.mktime(time.strptime(start_date,'%Y-%m-%d %H:%M:%S'))
            #end_time = time.mktime(time.strptime(end_date,'%Y-%m-%d %H:%M:%S'))

            if time_str>=self.start_time and time_str <= self.end_time :
                title = item.xpath("./div[@class='listTitle']/h3[@class='itemName']/a/text()").extract_first()
                hot_like = int(item.xpath("./div[@class='listRight']/div[@class='lrBot']/a[@class='zan']/em/text()").extract_first())
                hot_fav = int(item.xpath("./div[@class='listRight']/div[@class='lrBot']/a[@class='fav']/em/text()").extract_first())
                hot_comment = int(item.xpath("./div[@class='listRight']/div[@class='lrBot']/a[@class='comment']/text()").extract_first())
                author = item.xpath("./div[@class='listRight']/div[@class='lrTop']/span[2]/a/text()").extract_first()
               
                hot = hot_like + hot_fav + hot_comment
            
                pipe_item['title'] = title
                pipe_item['date'] = date_str
                pipe_item['hot'] = hot
                pipe_item['author'] = author
                yield pipe_item

        next_page = response.xpath("//ul[@class='pagination']/li[@class='pagedown']")

        if len(next_page) > 0 :
            next_page_url = next_page.xpath("./a/@href").extract_first()
            next_page_url = 'https:' + next_page_url
            yield scrapy.Request(next_page_url,callback=self.parse)

    


    def start_requests(self):   #项目启动，传入链接跟时间戳命令：scrapy crawl actHot -a url=https://www.smzdm.com/tag/t5veyee/post/ -a start_time=时间戳 -a end_time=时间戳 
        url = self.url
        self.start_time = int(self.start_time)
        self.end_time = int(self.end_time)
        yield scrapy.Request(url=url, callback=self.parse)




